{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def eval_logloss(y_true, y_prob):                      #评测函数\n",
    "    temp=pd.DataFrame({'true':y_true,'prob':y_prob})\n",
    "    sum=0\n",
    "    for ind in temp.index:\n",
    "        sum+=temp.loc[ind,'true']*np.log(temp.loc[ind,'prob'])+(1-temp.loc[ind,'true'])*np.log((1-temp.loc[ind,'prob']))\n",
    "    return -(sum/len(temp))\n",
    "def training(model,test,K,X,y,train_all,fortest=None,testresult=None):             #训练函数\n",
    "    from sklearn.model_selection import StratifiedKFold\n",
    "    skf=StratifiedKFold(n_splits = K,random_state = 1000,shuffle = True)\n",
    "    y_test_pred=0\n",
    "    y_valid_pred=0*y\n",
    "    train_test_pred=0\n",
    "    fit_model=[]\n",
    "    for i,(train_index,test_index) in enumerate(skf.split(X,y)):\n",
    "        y_train,y_valid=y.iloc[train_index],y.iloc[test_index]\n",
    "        x_train,x_valid=X.iloc[train_index],X.iloc[test_index]\n",
    "        print('Flod:',i,end=' ')\n",
    "        fitted_model=model.fit(x_train, y_train)#,categorical_feature=['user_gender_id']\n",
    "        pred= fitted_model.predict_proba(x_valid)[:,1]\n",
    "        y_valid_pred.iloc[test_index]=pred\n",
    "        if train_all==False:\n",
    "            train_test_pred+=fitted_model.predict_proba(fortest)[:,1]\n",
    "        y_test_pred+=fitted_model.predict_proba(test)[:,1]\n",
    "        fit_model.append(fitted_model)\n",
    "        import gc\n",
    "        del y_train,y_valid,x_train,x_valid\n",
    "        gc.collect()\n",
    "    print('logloss for full training set:',eval_logloss(y,y_valid_pred),end=' ')\n",
    "    if train_all==False:\n",
    "        train_test_pred/=K\n",
    "        print('logloss for full testing set:',eval_logloss(testresult,train_test_pred))\n",
    "    return y_test_pred/K,y_valid_pred\n",
    "def preproc(data):                         #对一些列进行加和除，例如总销售额，购买次数/展示次数\n",
    "    data['add_shop']=data['shop_score_service']+data['shop_score_delivery']+data['shop_score_description']+data['shop_review_positive_rate']\n",
    "    data['star_rshop']=data['shop_review_positive_rate']*data['shop_star_level']\n",
    "    data['total_sales']=data['item_price_level']*data['item_sales_level']\n",
    "    data['item_cv_pro']=data['item_collected_level']/data['item_pv_level']\n",
    "    data['item_sv_pro']=data['item_sales_level']/data['item_pv_level']\n",
    "    data.drop(['shop_score_description','shop_score_delivery','shop_score_service','shop_review_positive_rate',\\\n",
    "               'shop_star_level'],axis=1,inplace=True)\n",
    "    return data\n",
    "def category(tr,test):\n",
    "    # 类别形整数变量转换为类别变量\n",
    "    col_cato=list(tr.columns[tr.columns.str[-2:]=='id'])\n",
    "    h=['instance_id','user_id','item_id','context_id','shop_id','item_brand_id','item_city_id']\n",
    "    for i in h:\n",
    "        if i in col_cato:\n",
    "            col_cato.remove(i)\n",
    "    combine=[tr,test]\n",
    "    def int_to_category(df,col_cato):\n",
    "        for col in col_cato:\n",
    "            df[col]=df[col].astype('category')\n",
    "    for data in combine:\n",
    "        data=int_to_category(data,col_cato)\n",
    "    combine=[tr,test]\n",
    "    #类别形变量转换为指示性变量\n",
    "    for col in col_cato:\n",
    "        new_dummies=pd.get_dummies(tr[col],prefix=col)\n",
    "        tr=pd.concat([tr,new_dummies],axis=1)\n",
    "        tr.drop([col],axis=1,inplace=True)\n",
    "    for col in col_cato:\n",
    "        new_dummies=pd.get_dummies(test[col],prefix=col)\n",
    "        test=pd.concat([test,new_dummies],axis=1)\n",
    "        test.drop([col],axis=1,inplace=True)\n",
    "    print(len(tr.columns))\n",
    "    print(len(test.columns))\n",
    "    return tr,test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import gc\n",
    "import lightgbm as lgb\n",
    "train=pd.read_csv(r'D:\\Data\\IJCAI\\no_fixedday\\train_mispret.csv')\n",
    "test=pd.read_csv(r'D:\\Data\\IJCAI\\no_fixedday\\test_mispret.csv')\n",
    "new=pd.read_csv(r'D:\\Data\\IJCAI\\test.csv')\n",
    "ttest=test[['instance_id','item_id','user_id']].copy()\n",
    "testo=test.copy()\n",
    "K=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:26: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Flod: 0 Flod: 1 Flod: 2 logloss for full training set: 0.08917057649344934 logloss for full testing set: 0.08104010881428698\n"
     ]
    }
   ],
   "source": [
    "train_all=False\n",
    "test13=False\n",
    "# col=['com_pre_cate', 'com_pre_pro', 'com_pre_cateprob', 'com_pre_proprob','num_click','num_conversion', 'num_conversiontd', 'num_show', 'item_time_diff', 'day_item_diff', 'day_user_diff','is_trade','context_timestamp','context_id', 'user_id', 'item_id', 'shop_id', 'shop_score_delivery', 'shop_score_description', 'item_brand_id', 'item_sales_level', 'shop_score_service', 'shop_review_positive_rate', 'user_age_level', 'user_star_level', 'item_city_id', 'item_pv_level', 'shop_review_num_level', 'item_collected_level', 'item_category_list_0', 'context_page_id', 'item_price_level', 'user_occupation_id', 'user_gender_id', 'shop_star_level', 'predict_category_property_7908382889764677758', 'predict_category_property_8277336076276184272', 'predict_category_property_5799347067982556520', 'predict_category_property_509660095530134768', 'predict_category_property_7822717283490579102', 'predict_category_property_4144951645589726964', 'predict_category_property_7258015885215914736', 'predict_category_property_7492960463130085436', 'predict_category_property_2356297995131360540', 'predict_category_property_6007511169620956394', 'item_category_list_1', 'predict_category_property_2011981573061447208', 'predict_category_property_1950314698730389427', 'predict_category_property_5755694407684602296', 'predict_category_property_4864561375761257861', 'predict_category_property_6731593623692005614', 'predict_category_property_836752724084922533', 'predict_category_property_8710739180200009128', 'predict_category_property_7314150500379498593', 'predict_category_property_1771349742445680985', 'predict_category_property_6948560137918828076', 'predict_category_property_8426409705041090944', 'predict_category_property_1760164811125093110', 'predict_category_property_8896700187874717254', 'predict_category_property_4879721024980945592', 'predict_category_property_8257512457089702259', 'predict_category_property_2436715285093487584', 'predict_category_property_5912343598847406450', 'predict_category_property_2482352103497268539', 'predict_category_property_7730907179336883505']\n",
    "# col=['num_click', 'num_conversion', 'num_conversiontd', 'num_show', 'item_time_diff', 'day_item_diff', 'day_user_diff','is_trade','context_timestamp','context_id', 'user_id', 'item_id', 'shop_id', 'shop_score_delivery', 'shop_score_description', 'item_brand_id', 'item_sales_level', 'shop_score_service', 'shop_review_positive_rate', 'user_age_level', 'user_star_level', 'item_city_id', 'item_pv_level', 'shop_review_num_level', 'item_collected_level', 'item_category_list_0', 'context_page_id', 'item_price_level', 'user_occupation_id', 'user_gender_id', 'shop_star_level', 'predict_category_property_7908382889764677758', 'predict_category_property_8277336076276184272', 'predict_category_property_5799347067982556520', 'predict_category_property_509660095530134768', 'predict_category_property_7822717283490579102', 'predict_category_property_4144951645589726964', 'predict_category_property_7258015885215914736', 'predict_category_property_7492960463130085436', 'predict_category_property_2356297995131360540', 'predict_category_property_6007511169620956394', 'item_category_list_1', 'predict_category_property_2011981573061447208', 'predict_category_property_1950314698730389427', 'predict_category_property_5755694407684602296', 'predict_category_property_4864561375761257861', 'predict_category_property_6731593623692005614', 'predict_category_property_836752724084922533', 'predict_category_property_8710739180200009128', 'predict_category_property_7314150500379498593', 'predict_category_property_1771349742445680985', 'predict_category_property_6948560137918828076', 'predict_category_property_8426409705041090944', 'predict_category_property_1760164811125093110', 'predict_category_property_8896700187874717254', 'predict_category_property_4879721024980945592', 'predict_category_property_8257512457089702259', 'predict_category_property_2436715285093487584', 'predict_category_property_5912343598847406450', 'predict_category_property_2482352103497268539', 'predict_category_property_7730907179336883505']\n",
    "col=['com_pre_cate','hour','com_pre_pro','num_click','num_conversion', 'num_conversiontd', 'num_show', 'item_time_diff', 'newday_item_diff', 'newday_user_diff','item_id','item_city_id', 'item_price_level', 'item_sales_level', 'item_collected_level', 'item_pv_level', 'user_id', 'user_gender_id', 'user_age_level', 'user_occupation_id', 'user_star_level', 'context_id', 'context_timestamp', 'context_page_id', 'shop_id', 'shop_review_num_level', 'shop_review_positive_rate', 'shop_star_level', 'shop_score_service', 'shop_score_delivery', 'shop_score_description', 'is_trade']\n",
    "train['timestamp']=pd.to_datetime(train['context_timestamp'].copy(),unit='s')\n",
    "train['day']=train['timestamp'].copy().dt.day\n",
    "train=train[train['day']>17]\n",
    "tr=train[col].copy()\n",
    "tr=preproc(tr)\n",
    "col.remove('is_trade')\n",
    "test=testo.copy()\n",
    "test=test[col].copy()\n",
    "test=preproc(test)\n",
    "# tr,test=category(tr,test)\n",
    "if train_all==False:\n",
    "    fortest=tr[tr['context_timestamp']>1537718399]\n",
    "    np.random.seed(1000)\n",
    "    if test13==True:\n",
    "        lis=[]\n",
    "        for i in fortest.index:\n",
    "            if np.random.randint(3)==1:\n",
    "                lis.append(i)\n",
    "        fortest=fortest.loc[lis]\n",
    "    testresult=fortest['is_trade'].copy()\n",
    "    fortest.drop(['is_trade'],axis=1,inplace=True)\n",
    "    fortrain=tr[tr['context_timestamp']<=1537718399]\n",
    "    X=fortrain.drop(['is_trade'],axis=1)\n",
    "    y=fortrain['is_trade']\n",
    "else:\n",
    "    X=tr.drop(['is_trade'],axis=1)\n",
    "    y=tr['is_trade']\n",
    "# combine=[fortest,X,test]\n",
    "# for data in combine:\n",
    "#     data['timestamp']=pd.to_datetime(data['context_timestamp'].copy(),unit='s')\n",
    "#     data['weekday']=data['timestamp'].copy().dt.weekday\n",
    "#     data['hour']=data['timestamp'].copy().dt.hour\n",
    "#     data['day']=data['timestamp'].copy().dt.day\n",
    "#     data['minute']=data['timestamp'].copy().dt.minute\n",
    "# #     data['second']=data['timestamp'].copy().dt.second\n",
    "#     data['tm_hour']=data['hour'].copy()+data['minute'].copy()/60.0\n",
    "#     data.drop(['timestamp','context_timestamp'],axis=1,inplace=True)\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "skf=StratifiedKFold(n_splits = K,random_state = 1000,shuffle = True)\n",
    "model=lgb.LGBMClassifier(learning_rate=0.08,num_leaves=60, max_depth=7, n_estimators=80,min_child_samples=50,n_jobs=20,reg_lambda=1)\n",
    "y_test_pred,y_valid_pred=training(model,test,K,X,y,train_all,fortest,testresult)\n",
    "#0.08912307229856138 logloss for full testing set: 0.08104316694542328"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "no                   0.08318013595769923              0.09108811958036114<br/>\n",
    "0.08913559182213175 logloss for full testing set: 0.08095946639289714<br/>\n",
    "0.08983365980428494      0.0819244060584064 &nbsp;&nbsp;&nbsp;&nbsp;去掉17号&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;num_conversion<br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# importances=model.feature_importances_\n",
    "# indices=np.argsort(-importances)\n",
    "# print(importances[indices][:54])\n",
    "# print(X.columns[indices][:54])\n",
    "# list(X.columns[indices][:54])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_all=True\n",
    "col=['com_pre_cateprob', 'com_pre_proprob','num_click','num_conversion', 'num_conversiontd', 'num_show', 'item_time_diff', 'day_item_diff', 'day_user_diff','is_trade','context_timestamp','context_id', 'user_id', 'item_id', 'shop_id', 'shop_score_delivery', 'shop_score_description', 'item_brand_id', 'item_sales_level', 'shop_score_service', 'shop_review_positive_rate', 'user_age_level', 'user_star_level', 'item_city_id', 'item_pv_level', 'shop_review_num_level', 'item_collected_level', 'item_category_list_0', 'context_page_id', 'item_price_level', 'user_occupation_id', 'user_gender_id', 'shop_star_level', 'predict_category_property_7908382889764677758', 'predict_category_property_8277336076276184272', 'predict_category_property_5799347067982556520', 'predict_category_property_509660095530134768', 'predict_category_property_7822717283490579102', 'predict_category_property_4144951645589726964', 'predict_category_property_7258015885215914736', 'predict_category_property_7492960463130085436', 'predict_category_property_2356297995131360540', 'predict_category_property_6007511169620956394', 'item_category_list_1', 'predict_category_property_2011981573061447208', 'predict_category_property_1950314698730389427', 'predict_category_property_5755694407684602296', 'predict_category_property_4864561375761257861', 'predict_category_property_6731593623692005614', 'predict_category_property_836752724084922533', 'predict_category_property_8710739180200009128', 'predict_category_property_7314150500379498593', 'predict_category_property_1771349742445680985', 'predict_category_property_6948560137918828076', 'predict_category_property_8426409705041090944', 'predict_category_property_1760164811125093110', 'predict_category_property_8896700187874717254', 'predict_category_property_4879721024980945592', 'predict_category_property_8257512457089702259', 'predict_category_property_2436715285093487584', 'predict_category_property_5912343598847406450', 'predict_category_property_2482352103497268539', 'predict_category_property_7730907179336883505']\n",
    "train['timestamp']=pd.to_datetime(train['context_timestamp'].copy(),unit='s')\n",
    "train['day']=train['timestamp'].copy().dt.day\n",
    "tr=train[train['day']>17]\n",
    "tr=train[col].copy()\n",
    "tr=preproc(tr)\n",
    "col.remove('is_trade')\n",
    "test=testo.copy()\n",
    "test=test[col].copy()\n",
    "test=preproc(test)\n",
    "if train_all==False:\n",
    "    fortest=tr[tr['context_timestamp']>1537718399]\n",
    "    testresult=fortest['is_trade'].copy()\n",
    "    fortest.drop(['is_trade'],axis=1,inplace=True)\n",
    "    fortrain=tr[tr['context_timestamp']<=1537718399]\n",
    "    X=fortrain.drop(['is_trade'],axis=1)\n",
    "    y=fortrain['is_trade']\n",
    "else:\n",
    "    X=tr.drop(['is_trade'],axis=1)\n",
    "    y=tr['is_trade']\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "skf=StratifiedKFold(n_splits = K,random_state = 1000,shuffle = True)   \n",
    "import xgboost as xgb\n",
    "model=xgb.XGBClassifier(n_estimators=200,max_depth=6,objective=\"binary:logistic\",learning_rate=0.09,subsample=0.8,min_child_weight=8,\\\n",
    "                        colsample_bytree=0.9,scale_pos_weight=1.6,gamma=8,seed=100,reg_alpha=8,reg_lambda=1.3,nthread=19)\n",
    "y_test_pred1,y_valid_pred=training(model,test,K,X,y,train_all,fortest,testresult)\n",
    "# 0.0910413419281298 logloss for full testing set: 0.08102993576771603  600  6 0.09"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if train_all==True:\n",
    "    ttest['predicted_score']=(y_test_pred+y_test_pred1)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if train_all==True:\n",
    "    ttest[['instance_id','predicted_score']].to_csv(r'D:\\Data\\IJCAI\\model\\submission.csv',sep=' ',float_format='%.6f',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
